import json

from requests_html import HTMLSession
from util.redisQueue import RedisQueue
import threading
from constant.movieConfig import DYTT_NEW_MOVIE


class MovieListSpider(threading.Thread):

    def __init__(self, threadID, url):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.session = HTMLSession()
        self.r = RedisQueue('movie_url')
        self.url = url

    def run(self) -> None:
        self.getPages()

    def getPages(self):
        response = self.session.get(self.url)
        movie_last_pages = response.html.find(".co_content8 .x option", _encoding='gb2312')
        for page in movie_last_pages:
            # self.movie_last_pages.append(page.attrs['value'])
            print("正在打开网页https://www.dy2018.com{}".format(page.attrs['value']))
            self.openHtml("https://www.dy2018.com{}".format(page.attrs['value']))

    def openHtml(self, url):
        response = self.session.get(url)
        movie_items = response.html.find(".co_content8 ul a")
        movie = {}
        for movie_item in movie_items:
            for url in movie_item.absolute_links:
                movie['name'] = movie_item.text
                movie['url'] = url
                self.r.put(json.dumps(movie, sort_keys=True))



if __name__ == '__main__':
    MovieListSpider(1, DYTT_NEW_MOVIE).start()
